{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 其他辅助代码："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"Contains a variant of the densenet model definition.\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "slim = tf.contrib.slim\n",
    "\n",
    "\n",
    "def trunc_normal(stddev): return tf.truncated_normal_initializer(stddev=stddev)\n",
    "\n",
    "\n",
    "def bn_act_conv_drp(current, num_outputs, kernel_size, scope='block'):\n",
    "    current = slim.batch_norm(current, scope=scope + '_bn')\n",
    "    current = tf.nn.relu(current)\n",
    "    current = slim.conv2d(current, num_outputs, kernel_size,  padding='SAME',\n",
    "                          weights_initializer=trunc_normal(0.01), scope=scope + '_conv')\n",
    "    current = slim.dropout(current, scope=scope + '_dropout')\n",
    "    return current\n",
    "\n",
    "\n",
    "def block(net, layers, growth, scope='block'):\n",
    "    for idx in range(layers):\n",
    "        bottleneck = bn_act_conv_drp(net, 4 * growth, [1, 1],\n",
    "                                     scope=scope + '_conv1x1' + str(idx))\n",
    "        tmp = bn_act_conv_drp(bottleneck, growth, [3, 3],\n",
    "                              scope=scope + '_conv3x3' + str(idx))\n",
    "        net = tf.concat(axis=3, values=[net, tmp])\n",
    "    return net\n",
    "\n",
    "\n",
    "def bn_drp_scope(is_training=True, keep_prob=0.8):\n",
    "    keep_prob = keep_prob if is_training else 1\n",
    "    with slim.arg_scope(\n",
    "        [slim.batch_norm],\n",
    "            scale=True, is_training=is_training, updates_collections=None):\n",
    "        with slim.arg_scope(\n",
    "            [slim.dropout],\n",
    "                is_training=is_training, keep_prob=keep_prob) as bsc:\n",
    "            return bsc\n",
    "\n",
    "\n",
    "def densenet_arg_scope(weight_decay=0.004):\n",
    "    \"\"\"Defines the default densenet argument scope.\n",
    "\n",
    "    Args:\n",
    "      weight_decay: The weight decay to use for regularizing the model.\n",
    "\n",
    "    Returns:\n",
    "      An `arg_scope` to use for the inception v3 model.\n",
    "    \"\"\"\n",
    "    with slim.arg_scope(\n",
    "        [slim.conv2d],\n",
    "        weights_initializer=tf.contrib.layers.variance_scaling_initializer(\n",
    "            factor=2.0, mode='FAN_IN', uniform=False),\n",
    "        activation_fn=None, biases_initializer=None, padding='same',\n",
    "            stride=1) as sc:\n",
    "        return sc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### densenet的实现过程："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "整体网络结构：\n",
    "\n",
    "- 参照论文中 DenseNet-161 的结构，网络由 4 个 Dense Block 和 3 个 Transition Layer 构成。\n",
    "- 每个 Dense Block 依次包含 6、12、36、24 个 Layer，而每个 Layer 均包含一组 BN-ReLU-Conv-BN-ReLU-Conv 操作。其中第一个卷积层为 1×1（或称为Bottleneck Layer），目的是融合各个通道的特征，减少输入的 Feature Map 数量。第二个卷积层为 3×3，用于提取特征。在每个 Dense Block 内部，Feature Map 的大小不变。\n",
    "- Transition Layer 包含一个归一化层（Batch Normalization）、一个 1×1 的卷积层（用于减小 Feature Map 的数量）以及一个 2×2 的平均池化层（用于减小 Feature Map 的大小）。\n",
    "- 在进入第一个 Dense Block 之前，先进行一次 7×7、步长为2的卷积操作，以及一次 3×3、步长为2的最大池化；由最后一个 Dense Block 输出之后，先经过一次全局平均池化转化为特征向量，再经过归一化并激活后送入最后的全连接层（输出层）进行分类。\n",
    "\n",
    "\n",
    "对稠密连接的理解：\n",
    "\n",
    "- 在每个 Dense Block 内部，各 Layer 之间采取稠密链接的策略，在各个层之间建立直接连接，即每一个 Layer 将前面所有 Layer 的输出合并起来作为其输入。\n",
    "- 由于每个 Layer 都可以重新使用前面所有 Layer 计算出的特征图信息，提高了特征的利用效率，使模型更加紧凑，参数使用效率更高。\n",
    "- 这种连接方式使得每一层与输入信息和 loss 的连接更加紧密（途经更少的中间层），因此特征和梯度的传递会更加有效。一方面，特征的有效传递鼓励每一层都独立学习对分类有直接作用的特征；另一方面，梯度的有效传递可以减轻梯度消失现象，便于训练更深的网络。\n",
    "\n",
    "\n",
    "对 Growth 的理解:\n",
    "\n",
    "- Growth 用于控制每个卷积层输出的 Feature Map 数量，即网络宽度。由于每一个 Layer 的输出都要和前面所有 Layer 的输出叠加起来作为当前的 Feature Maps，如果一个 Dense Block 中有 L 个 Layer，每层输出 k 个 Feature Map，则该 Dense Block 最终输出 k×(L-1)+ k0 个 Feature Map。从这个角度看，Growth 实际上控制了每一个 Layer 为该 Dense Block 最终输出的特征图增加了多少信息。\n",
    "- 与 Growth 一起控制网络宽度的还有 Bottleneck Layer 和 Compression Rate。Bottleneck Layer 在每个 3×3 的卷积前面增加了一个 1×1 的卷积操作，将输入特征图的数量降为 Growth 的 4 倍；Compression Rate 在 Transition Layer 中增加 1×1 的卷积操作，将上一层输出的特征图降为原来的 Compression Rate 倍，这里设为 0.5。\n",
    "- 以上三者共同的作用使得网络变窄，参数减少，既增加了计算效率，又在一定程度上抑制了过拟合。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def densenet(images, num_classes=1001, is_training=False,\n",
    "             dropout_keep_prob=0.8,\n",
    "             scope='densenet'):\n",
    "    \"\"\"Creates a variant of the densenet model.\n",
    "\n",
    "      images: A batch of `Tensors` of size [batch_size, height, width, channels].\n",
    "      num_classes: the number of classes in the dataset.\n",
    "      is_training: specifies whether or not we're currently training the model.\n",
    "        This variable will determine the behaviour of the dropout layer.\n",
    "      dropout_keep_prob: the percentage of activation values that are retained.\n",
    "      prediction_fn: a function to get predictions out of logits.\n",
    "      scope: Optional variable_scope.\n",
    "\n",
    "    Returns:\n",
    "      logits: the pre-softmax activations, a tensor of size\n",
    "        [batch_size, `num_classes`]\n",
    "      end_points: a dictionary from components of the network to the corresponding\n",
    "        activation.\n",
    "    \"\"\"\n",
    "    growth = 48\n",
    "    compression_rate = 0.5\n",
    "\n",
    "    def reduce_dim(input_feature):\n",
    "        return int(int(input_feature.shape[-1]) * compression_rate)\n",
    "\n",
    "    end_points = {}\n",
    "\n",
    "    with tf.variable_scope(scope, 'DenseNet', [images, num_classes]):\n",
    "        with slim.arg_scope(bn_drp_scope(is_training=is_training,\n",
    "                                         keep_prob=dropout_keep_prob)) as ssc:\n",
    "            # pass\n",
    "            ##########################\n",
    "            # Put your code here.\n",
    "            # Convolution\n",
    "            end_point = 'Conv2d_7x7'\n",
    "            net = slim.conv2d(images, 2 * growth, [7, 7], stride=2, padding='SAME',\n",
    "                              weights_initializer=trunc_normal(0.01), scope=end_point)\n",
    "            end_points[end_point] = net\n",
    "            # Max_Pooling\n",
    "            end_point = 'MaxPool_3x3'\n",
    "            net = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', scope=end_point)\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Dense_block_1:\n",
    "            end_point = 'Block_1'\n",
    "            net = block(net, 6,  growth, scope=end_point)\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Transition_1:\n",
    "            end_point = 'Transition_1'\n",
    "            net = bn_act_conv_drp(net, reduce_dim(net), [1, 1], scope=end_point + '_con1x1')\n",
    "            net = slim.avg_pool2d(net, [2, 2], stride=2, padding='VALID', scope=end_point + '_avgpool')\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Dense_block_2:\n",
    "            end_point = 'Block_2'\n",
    "            net = block(net, 12, growth, scope=end_point)\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Transition_2:\n",
    "            end_point = 'Transition_2'\n",
    "            net = bn_act_conv_drp(net, reduce_dim(net), [1, 1], scope=end_point + '_con1x1')\n",
    "            net = slim.avg_pool2d(net, [2, 2], stride=2, padding='VALID', scope=end_point + '_avgpool')\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Dense_block_3:\n",
    "            end_point = 'Block_3'\n",
    "            net = block(net, 36, growth, scope=end_point)\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Transition_3:\n",
    "            end_point = 'Transition_3'\n",
    "            net = bn_act_conv_drp(net, reduce_dim(net), [1, 1], scope=end_point + '_con1x1')\n",
    "            net = slim.avg_pool2d(net, [2, 2], stride=2, padding='VALID', scope=end_point + '_avgpool')\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Dense_block_4:\n",
    "            end_point = 'Block_4'\n",
    "            net = block(net, 24, growth, scope=end_point)\n",
    "            end_points[end_point] = net\n",
    "\n",
    "            # Final pooling and classification\n",
    "            with tf.variable_scope('Logits'):\n",
    "                # Global average pooling.\n",
    "                net = slim.avg_pool2d(net, net.shape[1:3], stride=1, padding='VALID', scope='GlobalPool')\n",
    "                end_points['GlobalPool'] = net\n",
    "                \n",
    "                net = slim.batch_norm(net, scope='BN')\n",
    "                net = tf.nn.relu(net)\n",
    "\n",
    "                logits = slim.conv2d(net, num_classes, [1, 1], padding='VALID',\n",
    "                                     weights_initializer=trunc_normal(0.01), activation_fn=None,\n",
    "                                     normalizer_fn=None, scope='Conv2d_1x1')\n",
    "\n",
    "            end_points['Logits'] = logits\n",
    "            ##########################\n",
    "\n",
    "    return logits, end_points\n",
    "\n",
    "\n",
    "\n",
    "densenet.default_image_size = 224"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
